const inintDb = require('../utils/initDb_douban')();
const article = require('../models/articles');
const downloadPic = require('../models/imgDown');
const Crawler = require("crawler");
const path = require('path');

var fatherPath = path.resolve(__dirname,'..');
var c = new Crawler({
    maxConnections: 10,
    // This will be called for each crawled page
    callback: function (error, res, done) {
        if (error) {
            console.log(error);
        } else {
            var $ = res.$;
            // $ is Cheerio by default
            //a lean implementation of core jQuery designed specifically for the server

            var list = $('#content .article table').toArray();

            list.forEach(element => {
                var title = $(element).find('.pl2').text().trim().replace(/[ \n\r]/g, '');
                var author = $(element).find('.pl').text().trim().replace(/[ \n\r]/g, '').slice(0,$(element).find('.pl').text().trim().replace(/[ \n\r]/g, '').indexOf('/'));
                var introduction = $(element).find('.inq').text().trim().replace(/[ \n\r]/g, '');
                var img = $(element).find('img').attr('src');
                var imgFilePath = fatherPath + '/img/douban_img/' + img.slice(img.lastIndexOf('/') + 1);
                downloadPic(img, imgFilePath);
                article.create({ title: title, author: author, Introduction: introduction, imgFilePath: imgFilePath });
            })

        }
        done();
    }
});

// Queue just one URL, with default callback
for (let index = 0; index < 250; index += 25) {
    var URL = 'https://book.douban.com/top250?start=' + index;
    c.queue(URL);
}
